# coding=utf8
import requests
from bs4 import BeautifulSoup


def search_keyword(url):
    res = ["商品名,价格"]
    for item in get_elem(url):
        res.append(item)

    with open("data/fazada_shop_crawl.csv", 'w', encoding='utf-8-sig') as f:
        f.write("\n".join(res))


header = {
    'Connection': 'keep-alive',
    'Cookie': 'anon_uid=9c04388e-2f26-45a1-6943-46800cbd4495; AMCV_126E248D54200F960A4C98C6%40AdobeOrg=-1506950487%7CMCMID%7C61217194687048038510361361544008558313%7CMCAAMLH-1469273490%7C11%7CMCAAMB-1469273490%7CNRX38WO0n5BH8Th-nqAG_A%7CMCAID%7CNONE; browserDetection=eyJ0eXBlIjoiYnJvd3NlciIsIm5hbWUiOiJDaHJvbWUiLCJjc3NDbGFzcyI6ImNocm9tZSIsInZlcnNpb24iOiI1MyIsIm9zIjoibWFjIn0%3D; is_new_user=1; _lzdTracker=uhidnpadnwp0o3lxepwg66r; s_vnum=1500204692797%26vn%3D2; _vis_opt_s=1%7C; _vis_opt_exp_74_exclude=1; PHPSESSID_26cedaad9cc30104da500ad92159d7f5=eecf459675a82cf6750e5e5e4903d839; _tsm=m%3DDirect%2520%252F%2520Brand%2520Aware%253A%2520Typed%2520%252F%2520Bookmarked%2520%252F%2520etc%7Cs%3D%28none%29; search_type=Button; s_sq=%5B%5BB%5D%5D; userLanguageML=en; fd_location_entered=Not+filled+in; lzd_visitor_type=returning; lzd_first_purchase=Unknown; __utmx=93748224.fPWYWy8hQeO36xshPAWeZQ$0:2; __utmxx=93748224.fPWYWy8hQeO36xshPAWeZQ$0:1468679633:8035200; _vwo_uuid_v2=F4013E0BBE1C464A30769F3907827D34|929a39a6396e03daec4add21b5bd3e82; gpv_pn=search%3ANuskin%20Nu%20Skin%20Pharmanex%20Estera%20Phase%20III%20Womens%20Maintenance%20-%20Buy%20Nuskin%20Nu%20Skin%20Pharmanex%20Estera%20Phase%20III%20Womens%20Maintenance%20at%20Best%20Price%20in%20Malaysia%20%7C%20www.lazada.com.my; s_invisit=true; undefined_s=First%20Visit; s_cc=true; __utma=93748224.2124155511.1468676213.1468676213.1468676213.1; __utmb=93748224.20.9.1468679639508; __utmc=93748224; __utmz=93748224.1468676213.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=93748224.|37=Suggestion%20selected=0=1; _ga=GA1.3.2124155511.1468676213; _br_uid_2=uid%3D1380903303016%3Av%3D11.8%3Ats%3D1468668700449%3Ahc%3D9; session_counter=1; _vis_opt_test_cookie=1; s_ppvl=D%253Dch%2B%2522%253ANuskin%2520Nu%2520Skin%2520Pharmanex%2520Estera%2520Phase%2520III%2520Womens%2520Maintenance%2520-%2520Buy%2520Nuskin%2520Nu%2520Skin%2520Pharmanex%2520Estera%2520Phase%2520III%2520Womens%2520Maintenance%2520at%2520Best%2520Price%2520in%2520Malaysia%2520%257C%2520www.lazada.com.my%2522%2C46%2C46%2C676%2C817%2C676%2C1280%2C800%2C1%2CP; rr_rcs=eF4FwbsVgCAQBMCEyF7W53G_tQPbQCAwMFPrd6Ys9_dcY40MiAUj9zBX7jAHpLz9GJwntTaIasI2Juhm0OqefcwqjT9gwhDr; _ceg.s=oaewgv; _ceg.u=oaewgv; daily_visit_count=9|1468755331007; s_ppv=D%253Dch%2B%2522%253ANuskin%2520Nu%2520Skin%2520Pharmanex%2520Estera%2520Phase%2520III%2520Womens%2520Maintenance%2520-%2520Buy%2520Nuskin%2520Nu%2520Skin%2520Pharmanex%2520Estera%2520Phase%2520III%2520Womens%2520Maintenance%2520at%2520Best%2520Price%2520in%2520Malaysia%2520%257C%2520www.lazada.com.my%2522%2C60%2C46%2C880%2C817%2C676%2C1280%2C800%2C1%2CP',
    'Host': 'www.lazada.com.my',
    'Pragma': 'no-cache',
    'Referer': 'http://www.lazada.com.my/weight-management-products/?itemperpage=120&page=12',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.8 Safari/537.36',
}


def get_elem(url):
    n = 0
    while True:
        n += 1
        page_url = url.format(page=n)
        print(page_url)
        soup = BeautifulSoup(requests.get(page_url, headers=header).text, 'lxml')
        lis = soup.select(
            'div.component.component-product_list.product_list.grid.toclear > div')
        for li in lis:
            title = li.select_one('.product-card__name-wrap span')['title'].replace(',', '，').replace("\n", ' ')
            comm_num = li.select_one('.product-card__price').get_text().replace('RM ', '').replace(',', '')
            yield title + "," + str(comm_num)

        if n == 12:
            break


if __name__ == '__main__':
    search_keyword('http://www.lazada.com.my/weight-management-products/?itemperpage=120&page={page}')
